import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time,json,re,sys


url='http://www.jd.com'
taget_name="小米手机"
data_list = []
total_num=0 #总数据量
min_comment_num=20
# win firefox
web=webdriver.Firefox(executable_path='D:\\geckodriver.exe')
# 打开网页
web.get('http://www.jd.com')

# 输入搜索目标并点击
souhuo_input = web.find_element_by_css_selector("#key")
souhuo_input.send_keys(taget_name)
souhuo_button = web.find_element_by_css_selector(".button")
souhuo_button.click()

# 页面向下滑动10次
# js="var q=document.documentElement.scrollTop=10000"
for a in range(5):
    time.sleep(0.5)
    js = "var q=document.documentElement.scrollTop={}".format(1000+a*10000)
    web.execute_script(js)

# 获取页面信息
try:
    page_num=web.find_element_by_class_name('p-skip').find_element_by_tag_name('b').text
    # 翻页爬取

except:
    print("没有获取到页面数量信息！ error")
    web.close()
    sys.exit(0)

web.find_element_by_xpath("/html/body/div[6]/div[3]/div[2]/div[1]/div/div[1]/div[1]/div[1]/a[3]").click()

print("检测到目标数据共{}页".format(page_num))
for num in range(1,int(page_num)+1):
    time.sleep(1)
    flag=True #标记当前页面的数据需不需要爬，评论少就不爬
    print('正在爬取目标  ：{} 的第 {} 页商品'.format(taget_name,str(num)))
    page_input=web.find_element_by_css_selector('input.input-txt:nth-child(2)')
    page_input.clear() #先清空
    page_input.send_keys(num)
    page_input.send_keys(Keys.ENTER)

    # 页面向下滑动10次
    # js="var q=document.documentElement.scrollTop=10000"
    for a in range(5):
        time.sleep(0.5)
        js = "var q=document.documentElement.scrollTop={}".format(1000+a*10000)
        web.execute_script(js)

    shop_items=web.find_elements_by_class_name('gl-item')
    '''
    dict 类型
    {
       good_name:
       good_link:
       good_comment_num:
       good_price:
       good_shops:
    }
    '''
    # 获取tag标签内容
    for test_good in shop_items:
        temp_dict={}
        #获取标签内容
        try:
            good_name=test_good.find_element_by_class_name('p-name').find_element_by_tag_name('em').text
        except:
            good_name='未知'
        try:
            good_link=test_good.find_element_by_class_name('p-img').find_element_by_tag_name('a').get_attribute('href')
        except:
            good_link = '未知'
        try:
            commit_num = test_good.find_element_by_class_name('p-commit').text
            temp_num=re.sub('\D',"",commit_num.replace('万','0000'))
            if(int(temp_num)<min_comment_num):
                print(temp_num)
                print("检测到商品评论数量小于设定数量 {} 停止抓取 ".format(min_comment_num))
                flag=False
                break
            else:
                pass
        except:
            commit_num = '未知'
        try:
            good_price = test_good.find_element_by_class_name('p-price').text
        except:
            good_price = '未知'
        try:
            good_shop = test_good.find_element_by_class_name('p-shop').text
        except:
            good_shop = '未知'

        temp_dict['good_name']=good_name
        # print('正在爬取商品  {}  信息'.format(good_name))
        temp_dict['good_link']=good_link
        temp_dict['good_comment_num']=temp_num
        temp_dict['shops']=good_shop
        temp_dict['good_price']=good_price
        data_list.append(temp_dict)
        total_num+=1
    if(not flag):
        break

web.close()
print('一共抓取到{}条数据'.format(total_num))
with open("{}.json".format(taget_name),"w")as file:
    json.dump(obj=data_list,fp=file,ensure_ascii=False)
    print("{}.json".format(taget_name)," 数据保存完毕！")
